﻿using JiebaNet.Segmenter;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace TextEmbedding.Core
{
    public class JiebaNetTokenizer : ITokenizer
    {
        public string[] Tokenize(string text, bool shouldCutAll)
        {
            var segmenter = new JiebaSegmenter();
            return [.. segmenter.Cut(text, shouldCutAll)];
        }
    }
}
